#!/usr/bin/env python

################################################################################
#   Copyright 2010 Jason Hoover
#
#
#   This file is part of AutoCache.
#
#   AutoCache is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Lesser General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   AutoCache is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU Lesser General Public License
#   along with AutoCache.  If not, see <http://www.gnu.org/licenses/>.
################################################################################

################################################################################
# ac-common - Contains functions every program will need, including report 
# generators or the like.
# 
# Function                Description                        Returns
#
# ac_configurure(file)    Configuration parser               DICT of Settings 
# ac_stateinit()          Creates state directories/files.   Nothing
# ac_df(path)             Generates disk free percentage.    INT Percentage of disk free
# ac_du(path)             Runs du for specified path         INT Disk usage of path in MB
# ac_urltohash(str)       Converts a URLs to hash            STR MD5 Hash of second part.
# ac_urltofle(str)        Converts URLs into filenames       A filename derived from the URL
# ac_mknice()             Alias to os.nice with setting.     See os.nice.
# ac_log(str)             Wrapper of python logger           Nothing. See logging module docs.
# ac_compiletargets(list) Compiles targets into a regex.     A compiled regex.
################################################################################

import re
import os
import subprocess
import urlparse
import logging
import sys

from hashlib import md5

# Definitions here.
DEFAULT_CONFIGFILE =  "/etc/autocache.conf"


################################################################################
# ac_configure - Loads the configuration file into memory, parses it, and reads 
# those settings into ac_settings, which it then returns as it's result. All 
# settings should be specified, loaded, and parsed here. Everything else should
# be considered temporary. Should typically only be executed once per 
# exectutable at runtime. All of the other applications should load this.
################################################################################

def ac_configure ( config_location, cmdopts="" ):
    
    # Set up sane defaults.
    # WARNING This ac_settings is local. This function returns ac_settings 
    # which must be set globally!
    #
    # Definitions of each setting in brief:
    ###########################################
    # log_file        - Destination path to a log file.
    # log_format      - Format of the log entries.
    # log_level       - Logging level, specified in python's logger class.
    # du_exec         - Path to du.
    # dl_bwlimit      - Limit of downloads in kb.
    # dl_exec         - Path to wget.
    # dl_proxy        - Sets a target HTTP proxy.
    # dl_retry        - Number of times to retry a download.
    # dl_slots        - Number of simultanious downloads.
    # dl_timeout      - Socket/wget timeout in seconds.
    # nice            - Niceness for wgets and other background procs.
    # cache_dir       - Path to the cache directory.
    # cache_web       - http server path.
    # cache_pctlimit  - Max percentage of disk usage for cache.
    # cache_mblimit   - Max size of cache in MB.
    # cache_fmode     - Permissions in octal for cached files.
    # cache_dmode     - Permissions in octal for cache directories.
    # targets         - Path to targets file.
    # trust_long      - Seconds to trust fairly reliable objects.
    # trust_short     - Seconds to trust semi reliable objects.
    # state_dir       - Path to state storage.
    # concurrency     - Whether or not to perform concurrency to squid.
    # logger          - A logger object, created after parsing log_*.
    
    ac_settings = {
        "log_file"       :False,
        "log_level"      :0,
        "log_format"     :"%(asctime)s [%(process)d:%(threadName)s] %(levelname)s %(funcName)s: %(message)s",
        "du_exec"        :"du",
        "dl_exec"        :"",
        "dl_bwlimit"     :0,
        "dl_program"     :"wget",
        "dl_retry"       :5,
        "dl_slots"       :1,
        "dl_timeout"     :90,
        "dl_proxy"       :"",
        "nice"           :0,
        "cache_dir"      :"",
        "cache_web"      :"",
        "cache_pctlimit" :90,
        "cache_mblimit"  :0,
        "cache_fmode"    :0744,
        "cache_dmode"    :0755,
        "targets"        :"",
        "trust_long"     :1000,
        "trust_short"    :60,
        "state_dir"      :"",
        "concurrency"    :False,
        "logger"         :False,
        }

    # Define words that imply true or false so they can be used in the config
    # if necessary.
    true_false = {
        "true"  : True,
        "1"     : True,
        "yes"   : True,
        "0"     : False,
        "no"    : False,
        "false" : False,
        }

    # Define the possible levels here.
    log_levels = {
        'debug'    : logging.DEBUG,
        'info'     : logging.INFO,
        'warning'  : logging.WARNING,
        'error'    : logging.ERROR,
        'critical' : logging.CRITICAL,
        }

    # Define valid downloaders here.
    dl_programs = ( 'aria2c', 'curl', 'fetch', 'wget', )

    # Define required configuration entries:
    required_settings = ( 
        'cache_dir',
        'cache_mblimit',
        'cache_web', 
        'targets', 
        'state_dir', 
        )

    # Open and read the configuration file object. Die if that fails.
    try:
        config_file = open( config_location )
        config_contents = {} 

        # Loop through until you hit EOFError.
        for lastline in config_file:

            # Read the file into a dict. Read line by line, strip newlines, 
            # make and split at ='s.
            lastline = lastline.rstrip().split('=')

            # Ignore comments and runts.
            if lastline[0].lstrip().startswith(('#',';')) or lastline[0] == '':
                continue

            # Permit continuation of lines with preceeding whitespace.
            elif lastline[0][0].isspace():
                config_contents[lastoption] += lastline[0]
                continue

            # Ignore lines with no options.
            elif len(lastline) < 2:
                print("WARNING - Badly formed configuration line: " + str(lastline))
                continue

            # Normalize the config item names. It's important to strip the
            # surrounding whitespace around the options and settings at this 
            # point, or else the options will fail to be parsed.
            config_contents[lastline[0].lower().strip()] = lastline[1].strip()
            lastoption=lastline[0].lower().strip()

    except (OSError,IOError):
        raise OSError ("Configuration file was unreadable.")
 

    # Check for all the required settings so we don't have to do it later.
    for each in required_settings:
        if each not in config_contents:
            raise EnvironmentError("Missing required setting in configuration: " 
                                   + each)


    # Create the logger object regardless of if it's configured. We'll be 
    # using it no matter if it's configured or not!
    ac_settings["logger"] = logging.getLogger("ac_log")

    # Set the log file destination by adding a handler to ac_settings["logger"]
    if config_contents.has_key("log_file"):

        # Make sure we can write the file. Open it in append mode to not wipe
        # it out.
        try:
            file(config_contents["log_file"], 'a').close()

        except:
            raise OSError("Invalid setting for 'log_file'. Must be a writable file.")

        ac_log_handler=logging.FileHandler(config_contents["log_file"])

    # If no log destination is set, set it to stderr.
    else:
        ac_log_handler=logging.StreamHandler()


    # Formatting configuration must be done before the handler is added to
    # the logger.
    if config_contents.has_key("log_format") \
       and config_contents["log_format"] > "":

        ac_log_handler.setFormatter(logging.Formatter(config_contents["log_format"]))

    else:

        ac_log_handler.setFormatter(logging.Formatter(ac_settings["log_format"]))
        
    # Add a log file handler.
    ac_settings["logger"].addHandler(ac_log_handler)

    # If there's no log level set, set it to everything higher than debug.
    if config_contents.has_key("log_level"):

        try:
            # And then set the correct level. Note that we use NOTSET in the
            # event that there is no log level set, which by default is
            # (i think) everything but debug.
            ac_settings["logger"].setLevel(
                log_levels.get(config_contents["log_level"].lower())
                )

        except: 
            raise ValueError("Invalid setting for 'log_level'. See README for valid options.")

    else:
        ac_settings["logger"].setLevel(logger.NOTSET)


    # Set the dl_program.
    if config_contents.has_key("dl_program") \
       and config_contents["dl_program"] in dl_programs:
            ac_settings["dl_program"]=config_contents["dl_program"]


    # Set dl_exec if set.
    if config_contents.has_key("dl_exec"):
 
           ac_settings["dl_exec"] = config_contents["dl_exec"]

    # If it's not set explicitly, default to dl_program.
    else:
        ac_settings["dl_exec"] = ac_settings["dl_program"]


    # Regardless of what it's set to, we need to be able to execute the 
    # dl_program.
    try:
        subprocess.call(ac_settings["dl_program"], stdout=subprocess.PIPE)

    except OSError:
        raise OSError ("Invalid setting for 'dl_exec'. Must point to executable. Was: "
                       + ac_settings["dl_exec"])


    # Same for du. (Maybe I should make a function for this.. >___>)
    if config_contents.has_key("du_exec"):

            ac_settings["du_exec"]=config_contents["du_exec"]


    # Verify the the executability of du.
    try:
        subprocess.call([ac_settings["du_exec"], "/dev/null"], 
                        stdout=subprocess.PIPE)

    except OSError:
        raise OSError("Invalid setting for 'du_exec'. Must point to executable. Was: "
                      + ac_settings["du_exec"])

    # Set the dl_bwlimit value in kilobytes.
    # If not set, leave the default value and do no further processing.
    # If higher than 0, less than 1 terabyte per second, raise one ourselves.
    if not config_contents.has_key("dl_bwlimit"):
        pass

    elif config_contents["dl_bwlimit"].isdigit() \
         and 0 < int(config_contents["dl_bwlimit"]) < 1000000001:

            ac_settings["dl_bwlimit"] = int(config_contents["dl_bwlimit"])

    else:
        raise ValueError("Invalid setting for 'dl_bwlimit'. Must be a positive integer between 1 and 1000000000, or 0 for unlimited. Was : " + 
                         config_contents["dl_bwlimit"])


    # Set the dl_proxy
    if config_contents.has_key("dl_proxy"):
        ac_settings["dl_proxy"] = config_contents["dl_proxy"]


    # Max disk usage percent..
    if not config_contents.has_key("cache_pctlimit"):
        pass

    elif config_contents["cache_pctlimit"].isdigit() \
         and 0 < int(config_contents["cache_pctlimit"]) < 101:

            ac_settings["cache_pctlimit"]=int(config_contents["cache_pctlimit"])

    else:
        raise ValueError("Invalid setting for 'cache_pctlimit'. Must be a positive integer between 1 and 100. Was: "
                         + config_contents["cache_pctlimit"])


    # Download slots.
    if not config_contents.has_key("dl_slots"):
        pass

    elif config_contents["dl_slots"].isdigit() \
         and 0 < int(config_contents["dl_slots"]) < 101:

            ac_settings["dl_slots"]=int(config_contents["dl_slots"])

    else:
        raise ValueError("Invalid setting for 'dl_slots'. Must be a poaitive integer between 1 and 100. Was: "
                         + config_contents["dl_slots"])

    # Max disk usage in MB
    if config_contents.has_key("cache_mblimit")       \
       and config_contents["cache_mblimit"].isdigit() \
       and int(config_contents["cache_mblimit"]) > 0:

           ac_settings["cache_mblimit"]=(int(config_contents["cache_mblimit"]) 
                                         * 1024 * 1024 )

    else:

        raise ValueError("Invalid setting for 'cache_mblimit'. Must be a positive integer greater than 0. Was: " + 
                         config_contents["cache_pctlimit"])
        
    # Remember to use -or- since these are bit comparisons.
    if os.path.isdir(config_contents["cache_dir"]) \
       and os.access(config_contents["cache_dir"], os.R_OK | os.W_OK):

            # Add a trailing slash now so I don't have to enter it in the code
            # 15 fucking times.
            ac_settings["cache_dir"]=config_contents["cache_dir"] + "/"

    else:
        raise OSError("Invalid setting for 'cache_dir'. Must be a writable, readable directory. Was: " +
                      config_contents["cache_dir"])

    # Set the state_dir
    if os.access(config_contents["state_dir"], os.R_OK | os.W_OK) \
       and os.path.isdir(config_contents["state_dir"]):

            # Add a trailing slash now so I don't have to enter it in the code
            # 15 fucking times.
            ac_settings["state_dir"] = config_contents["state_dir"] + "/"

    else:

        raise OSError("Invalid setting for 'state_dir'. Must be a writable, readable directory. Was: " + 
                      config_contents["state_dir"])


    # Add the targets. Split because compiletargets expects a list.
    ac_settings["targets"]=config_contents["targets"].split()


    # Configure cache_web, the directed cache target.
    if urlparse.urlparse(config_contents["cache_web"]).scheme != "http":

        raise ValueError("Invalid setting for 'cache_web'. Only http URLs are supported. Was: "
                         + config_contents["cache_web"])
            
    elif urlparse.urlparse(config_contents["cache_web"]).netloc == "":

        raise ValueError("Invalid setting for 'cache_web'. Unable to read the hostname. Was: "
                         + config_contents["cache_web"])

    else:

        ac_settings["cache_web"] = config_contents["cache_web"]


    # Lastly, validate nice, leave default if unset. From nice(1), should be
    # between -20 and 19.
    if not config_contents.has_key("nice"):

        pass
 
    elif config_contents["nice"].isdigit() \
         and -21 < int(config_contents["nice"]) < 20:

                ac_settings["nice"] = int(config_contents["nice"])

    else:

        raise ValueError("Invalid setting for 'nice'. Must be an integer between -20 and 19. Was: "
                         + config_contents["nice"])


    # Configure concurrency - If set.
    if config_contents.has_key("concurrency"):

        ac_settings["concurrency"] = true_false.get(
                                         config_contents["concurrency"].lower(),
                                         False)


    # Set up cache file mode settings.
    if config_contents.has_key("cache_fmode"):

        try:

            # Limit the size of the value to fit into permissions.
            if 0 < int(config_contents["cache_fmode"]) < 10000:
                raise ValueError

            # Turn the STR generated by the config reader into a base 8 INT (a
            # decimalized octal) so that it can be given to os.chmod.
            ac_settings["cache_fmode"] = int(config_contents["cache_fmode"], 8)

        except ValueError:
            raise ValueError("Invalid setting for 'cache_fmode'. Must be octal permissions. Was: " 
                             + config_contents["cache_fmode"] )


    # Set up cache directory mode settings.
    if config_contents.has_key("cache_dmode"):

        try:    

            # Limit the size of the value to fit into permissions.
            if 0 < int(config_contents["cache_dmode"]) < 10000:
                raise ValueError

            # Turn the STR generated by the config reader into a base 8 INT (a
            # decimalized octal) so that it can be given to os.chmod.
            ac_settings["cache_dmode"] = int(config_contents["cache_dmode"], 8)

        # A ValueError means the octal wasn't.
        except ValueError:
            raise ValueError("Invalid setting for 'cache_dmode'. Must be octal permissions. Was: " 
                             + config_contents["cache_dmode"] )


    # Set up the default timeout value for network I/O
    if not config_contents.has_key("dl_timeout"):

        pass

    elif config_contents["dl_timeout"].isdigit() \
         and 0 < int(config_contents["dl_timeout"]) < 1001:

            ac_settings["dl_timeout"] = int(config_contents["dl_timeout"])

    else:

        raise ValueError("Invalid setting for 'dl_timeout'. Must be a positive integer between 1 and 1000. Was: " + 
                         config_contents["dl_timeout"])

    # Configure retries for downloaders.
    if not config_contents.has_key("dl_retry"):

        pass

    elif config_contents["dl_retry"].isdigit() \
         and 0 < int(config_contents["dl_retry"]) < 1001:

             ac_settings["dl_retry"] = int(config_contents["dl_retry"])

    else:

        raise ValueError("Invalid setting for 'dl_retry'. Must be a positive integer between 1 and 1000. Was: " 
                         + config_contents["dl_retry"])


    # All done.
    return ac_settings

################################################################################
# ac_stateinit - configures the state DIR. Extracted from the original 
# configuration utility because there's sometimes a need to call this from other
# functions if the statedir is wiped or fails.
################################################################################

def ac_stateinit():

    # Let's build a list of files and directories that must exist.
    files = ( 
        "queue", 
        "lastcleanup",
        "cachesize", 
        )

    # If we have to create sub directories, make sure to write them IN ORDER.
    dirs = (
        "download", 
        "download/active",
        "download/locks",
        )

    for each in dirs:

        each = ac_settings["state_dir"] + each

        try:

            if not os.access(each, os.R_OK | os.W_OK) or not os.path.isdir(each):
                os.mkdir(each)
                ac_log.info("Created state directory: " + each)

        except OSError:

            # Shouldn't ever happen. At this point, the state_dir should exist
            #  and writable. Maybe it got replaced with a file or something?
            ac_log.critical("Could not set up state directory: " + each)
            raise OSError("Could not set up state directory: " + each)


    for each in files:

        each = ac_settings["state_dir"] + each

        try:

            if not os.access(each, os.R_OK | os.W_OK) or not  \
               os.path.isfile(each):
                    file(each, 'a').close()
                    ac_log.info("Needed to create state file: " + each)

        except IOError:

            # Again, should never happen.
            ac_log.critical("Could not read/write/create the state file: "
                            + each)
            raise IOError("Could not read/write/create the state file: "
                          + each)


    return


################################################################################
# ac_df - lambda function, returning free disk space of path in percent.
################################################################################

ac_df = lambda path: int(float(os.statvfs(path).f_bfree) / 
                         float(os.statvfs(path).f_blocks) *
                         100)


################################################################################
# ac_du - A terrible, complex lambda function, returning disk usage in bytes of 
# a given path. Takes a long time to execute because du is VERY I/O heavy, so it 
# should be used SPARINGLY.
#
# A quick rundown of how this works:
# -Open a subprocess for du, summary (-s) in bytes (-B 1), path.
# -Put stdout and stderr into the communicate object.
# -Start the program with ac_mknice
# -Select the stdout, which is 0
# -Split the stdout by \t.
# -Return the first chunk of the split output.
#
################################################################################

ac_du = lambda path: int( 
    subprocess.Popen(
        (ac_settings["du_exec"],
         "-s",
         "-B 1",
         path,),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        preexec_fn=ac_mknice,
        ).communicate()[0].split("\t")[0]
    )


################################################################################
# ac_urltohash - Another simple lambda function that turns a URL into an MD5 
# hash. This is what will be stored in the cache_dir. Standalone so that the 
# generator can be easily modified.
################################################################################

ac_urltohash = lambda url: md5(urlparse.urlparse(url).path).hexdigest()


################################################################################
# ac_urltofile - A short, simple, stupid lambda which returns a filename for a 
# given url. Similar to urltohash, this is so that we can change the way 
# filenames are generated throughout the whole program easily.
################################################################################

ac_urltofile = lambda url: str(url).split('/')[-1]


################################################################################
# ac_mknice - Make a subprocess nice. Called as a preexec_fn for wget and du.
################################################################################

ac_mknice = lambda: os.nice(ac_settings["nice"])


################################################################################
# ac_compiletargets - Uses re (regexp) lib and makes precompiled regexps and
# appends them to a list. this list is then checked against by the main
# function. I originally wanted to have seperate regexps per domain, but this
# turned out to be more efficent in the end.
################################################################################

def ac_compiletargets():

    compiled = ""

    for target in ac_settings['targets']:

        try:

            # flags=re.I makes the regexps case insensitive, because URLs
            # aren't a case-sensitive thing.
            re.compile(target, flags=re.I)
            ac_log.debug("Adding successfully compiled target: " + target)
            compiled = compiled + target + "|"

        except re.error:

            ac_log.error("Error compiling regular the expression: " + target)

    # Make the regex.
    try:

        # Remove the last | so we don't have a null match.
        compiled = compiled[:-1]
        ac_log.debug("Going to compile the assembled target regex: " + compiled)
        compiled = re.compile(compiled, flags=re.I)

    except re.error:
        ac_log.error("Failed to compile targets. Check syntax. " 
                     + compiled)

    return compiled


################################################################################
# The functions below are executed upon import.
################################################################################

try:
    # Import the config specified on the command line.
    if "-c" in sys.argv: 
        ac_settings=ac_configure(sys.argv[(sys.argv.index('-c') + 1)])

    elif "--config" in sys.argv:
        ac_settings=ac_configure(sys.argv[(sys.argv.index('--config') + 1)])

    elif os.path.isfile(DEFAULT_CONFIGFILE):
        ac_settings=ac_configure(DEFAULT_CONFIGFILE)

    else:
        print("Default configuration file (" + DEFAULT_CONFIGFILE 
              + ") not found. Please specify one using -c or --config <filename>.")
        sys.exit(1) 

    ac_log = ac_settings["logger"]

except IOError:
    raise IOError("Could not read specified configuration file.")


